package com.zouxian.processor;
import java.util.ArrayList;
import java.util.List;

import com.zouxain.scheduler.SpikeFileCacheQueueScheduler;
import com.zouxian.pipeline.MyFilePipeline;
import com.zouxian.pojo.News;
import com.zouxian.util.StringUtils;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.PhantomJSDownloader;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
//环球网，+ PhantomJSDownloader爬取
public class MyProcessor7  implements PageProcessor{
	//配置
	private Site site = Site.me().setRetryTimes(3).setSleepTime(100);
	private static int count=0;
	public Site getSite() {
		// TODO Auto-generated method stub
		return site;
	}
	
	public void process(Page page) {
		News news=new News();
		Boolean flag=page.getUrl().regex("http://[a-z]+.huanqiu.com/[0-9 a-z]+/[0-9 -]+/[0-9]+.html").match();
		if(!flag){
			page.addTargetRequests(page.getHtml().links().regex("http://[a-z]+.huanqiu.com/[0-9 a-z]+/[0-9 -]+/[0-9]+.html").all());
		}
		count++;
		try {
			Thread.sleep(5000);
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
		//获取编码
		String htmlCode = page.getCharset();
		news.setHtmlCode(htmlCode);
		//获取url
		String Url=page.getUrl().get();
		//page.putField("url", url);
		news.setUrl(Url);
		//获取标题
		String title=page.getHtml().$(".tle","text").toString();
		if(title==null){
			page.setSkip(true);
			
		}
		//page.putField("title", title);
		news.setTitle(title);
		//获取类型
		String opicType=page.getHtml().$(".nav_left a","text").all().toString();
		news.setOpicType(opicType);
		//获取关键字
		/*String keyword=page.getHtml().$(".label__link-wrapper","text").toString();
		page.putField("keyword", keyword);*/
		//获取日期
		String GetTime=page.getHtml().$(".la_t_a","text").toString();
		//page.putField("data", putTime);
		news.setGetTime(GetTime);
		//获取来源
		String Source=page.getHtml().$(".la_t_b a","text").toString();
		//page.putField("resource", resource);
		news.setSource(Source);
		//获取作者
		String author=page.getHtml().$(".la_t_c span","text").toString();
		//page.putField("author", author);
		news.setAuthor(author);
		//获取参与人数
		String pinlunNum=page.getHtml().$(".participate var:first-child","text").toString();
		//page.putField("canyu", canyu);
		news.setPinlunNum(pinlunNum);
		//获取正文
		String Txt=page.getHtml().$(".la_con p","text").all().toString();
		Txt= StringUtils.replaceBlank(Txt);
		news.setTxt(Txt);
		//page.putField("content", content);
		//获取分类板块
		String SubColum=page.getHtml().$(".nav_left a","text").all().toString();
		page.putField("title", title);
		page.putField("TitleCh", news.getTitleCh());
		page.putField("Title_Score", news.getTitle_Score());
		page.putField("articalKeyWords", news.getArticalKeyWords());
		page.putField("Txt", news.getTxt());
		page.putField("TxtCh", news.getTxtCh());
		page.putField("Txt_Score", news.getTxt_Score());
		page.putField("Url", news.getUrl());
		page.putField("Summary", news.getSummary());
		page.putField("SummaryCh", news.getSummaryCh());
		page.putField("TxtType", news.getTxtType());
		page.putField("GetTime", news.getGetTime());
		page.putField("opicType", news.getOpicType());
		page.putField("Source", news.getSource());
		page.putField("isdown", news.getIsdown());
		page.putField("relateNewsNumber", news.getRelateNewsNumber());
		page.putField("KeyWords", news.getKeyWords());
		page.putField("schoolName", news.getSchoolName());
		page.putField("country", news.getCountry());
		page.putField("Polar", news.getPolar());
		page.putField("author", news.getAuthor());
		page.putField("language", news.getLanguage());
		page.putField("htmlCode", htmlCode);
		page.putField("semanticValue", news.getSemanticValue());
		page.putField("mediaIntroduction", news.getMediaIntroduction());
		page.putField("zuanfaNum", news.getZuanfaNum());
		page.putField("pinlunNum", news.getPinlunNum());
		page.putField("shouchangNum", news.getSchoolName());
		page.putField("txtSeg", news.getTxtSeg());
		
	}
	
	//.setDownloader(new SeleniumDownloader("C:\\Users\\zx\\Downloads\\chromedriver.exe"))
	public static void main(String[] args) {
		SpikeFileCacheQueueScheduler file=new SpikeFileCacheQueueScheduler("D:\\test\\url");
		long startTime, endTime;
		System.out.println("开始爬取...");
		startTime = System.currentTimeMillis();
		Spider.create(new MyProcessor7()).addUrl("https://www.huanqiu.com").setScheduler(file).addPipeline(new ConsolePipeline()).addPipeline(new MyFilePipeline()).setDownloader(new PhantomJSDownloader("D:\\developTools\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe", "D:\\crawl.js")).thread(5).run();
		endTime = System.currentTimeMillis();
		System.out.println("爬取结束，耗时约" + ((endTime - startTime) / 1000)
				+ "秒，抓取了" + count + "条记录");
		
	}

}
